{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "file = 'data/add_cargo.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "outputs": [
    {
     "data": {
      "text/plain": "       Unnamed: 0 bad_case_name core_word ignore_word           reason_text  \\\n0               0         苹果123        苹果         123  包含数量、价格、包装、尺寸、装卸货等信息   \n1               1     需24米内径装木方        木方       24米内径  包含数量、价格、包装、尺寸、装卸货等信息   \n2               2      9吨铁皮7吨岩棉        铁皮          9吨  包含数量、价格、包装、尺寸、装卸货等信息   \n3               3       在丹江口市边装                        包含数量、价格、包装、尺寸、装卸货等信息   \n4               4          袋装化肥                        包含数量、价格、包装、尺寸、装卸货等信息   \n...           ...           ...       ...         ...                   ...   \n54681       54681    溧阳上兴镇到芜湖九江            溧阳上兴镇到芜湖九江                   无货名   \n54682       54682      电瓶车蛇皮袋4袋                                        多个货物   \n54683       54683        泡沫板和型钢                                        多个货物   \n54684       54684          电线钻头                                        多个货物   \n54685       54685         小电机纸箱       小电机          纸箱  包含数量、价格、包装、尺寸、装卸货等信息   \n\n      reason_id correct_word  is_valid new_core_word  \n0          1001                      1            苹果  \n1          1001                      1            木方  \n2          1001                      1            铁皮  \n3          1001                      1                \n4          1001                      1            化肥  \n...         ...          ...       ...           ...  \n54681      1003                      1                \n54682      1002                      1           电瓶车  \n54683      1002                      1           泡沫板  \n54684      1002                      1            电线  \n54685      1001                      1           小电机  \n\n[54686 rows x 9 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Unnamed: 0</th>\n      <th>bad_case_name</th>\n      <th>core_word</th>\n      <th>ignore_word</th>\n      <th>reason_text</th>\n      <th>reason_id</th>\n      <th>correct_word</th>\n      <th>is_valid</th>\n      <th>new_core_word</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0</td>\n      <td>苹果123</td>\n      <td>苹果</td>\n      <td>123</td>\n      <td>包含数量、价格、包装、尺寸、装卸货等信息</td>\n      <td>1001</td>\n      <td></td>\n      <td>1</td>\n      <td>苹果</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1</td>\n      <td>需24米内径装木方</td>\n      <td>木方</td>\n      <td>24米内径</td>\n      <td>包含数量、价格、包装、尺寸、装卸货等信息</td>\n      <td>1001</td>\n      <td></td>\n      <td>1</td>\n      <td>木方</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>2</td>\n      <td>9吨铁皮7吨岩棉</td>\n      <td>铁皮</td>\n      <td>9吨</td>\n      <td>包含数量、价格、包装、尺寸、装卸货等信息</td>\n      <td>1001</td>\n      <td></td>\n      <td>1</td>\n      <td>铁皮</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>3</td>\n      <td>在丹江口市边装</td>\n      <td></td>\n      <td></td>\n      <td>包含数量、价格、包装、尺寸、装卸货等信息</td>\n      <td>1001</td>\n      <td></td>\n      <td>1</td>\n      <td></td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>4</td>\n      <td>袋装化肥</td>\n      <td></td>\n      <td></td>\n      <td>包含数量、价格、包装、尺寸、装卸货等信息</td>\n      <td>1001</td>\n      <td></td>\n      <td>1</td>\n      <td>化肥</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>54681</th>\n      <td>54681</td>\n      <td>溧阳上兴镇到芜湖九江</td>\n      <td></td>\n      <td>溧阳上兴镇到芜湖九江</td>\n      <td>无货名</td>\n      <td>1003</td>\n      <td></td>\n      <td>1</td>\n      <td></td>\n    </tr>\n    <tr>\n      <th>54682</th>\n      <td>54682</td>\n      <td>电瓶车蛇皮袋4袋</td>\n      <td></td>\n      <td></td>\n      <td>多个货物</td>\n      <td>1002</td>\n      <td></td>\n      <td>1</td>\n      <td>电瓶车</td>\n    </tr>\n    <tr>\n      <th>54683</th>\n      <td>54683</td>\n      <td>泡沫板和型钢</td>\n      <td></td>\n      <td></td>\n      <td>多个货物</td>\n      <td>1002</td>\n      <td></td>\n      <td>1</td>\n      <td>泡沫板</td>\n    </tr>\n    <tr>\n      <th>54684</th>\n      <td>54684</td>\n      <td>电线钻头</td>\n      <td></td>\n      <td></td>\n      <td>多个货物</td>\n      <td>1002</td>\n      <td></td>\n      <td>1</td>\n      <td>电线</td>\n    </tr>\n    <tr>\n      <th>54685</th>\n      <td>54685</td>\n      <td>小电机纸箱</td>\n      <td>小电机</td>\n      <td>纸箱</td>\n      <td>包含数量、价格、包装、尺寸、装卸货等信息</td>\n      <td>1001</td>\n      <td></td>\n      <td>1</td>\n      <td>小电机</td>\n    </tr>\n  </tbody>\n</table>\n<p>54686 rows × 9 columns</p>\n</div>"
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "origin_df = pd.read_csv(file,sep=',')\n",
    "origin_df.fillna(\"\",inplace=True)\n",
    "origin_df"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [
    {
     "data": {
      "text/plain": "                    错误码     个数\n0                   无货名  18771\n1  包含数量、价格、包装、尺寸、装卸货等信息  16694\n2                  多个货物  15607\n3                 错字或拼音   4496\n4                    货类    601\n5           危险品/危化品/违禁词    111",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>错误码</th>\n      <th>个数</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>无货名</td>\n      <td>18771</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>包含数量、价格、包装、尺寸、装卸货等信息</td>\n      <td>16694</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>多个货物</td>\n      <td>15607</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>错字或拼音</td>\n      <td>4496</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>货类</td>\n      <td>601</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>危险品/危化品/违禁词</td>\n      <td>111</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = origin_df['reason_text'].str.split('|', expand=True).stack()\n",
    "c = a.value_counts()\n",
    "d = pd.DataFrame(data=c,columns=[\"个数\"]).reset_index().rename(columns={\"index\":\"错误码\"})\n",
    "d"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "outputs": [
    {
     "data": {
      "text/plain": "     core_word                                      bad_case_name\n0               在丹江口市边装$$袋装化肥$$包装饲料$$散装粮食$$南瓜卸货$$散石油焦公司一手卸车现金$...\n1        115挖机                                          115挖机机马上走\n2        120挖机                             120挖机机马上走$$120挖机挖机机马上走\n3        125每吨                                           好铁沙125每吨\n4       1304车头                                           1304车头1台\n...        ...                                                ...\n4630      龙船摆件                                             3米龙船摆件\n4631        龙虾                                               厢装龙虾\n4632       龙虾尾                                              龙虾尾冷运\n4633      龙门吊梁                                           龙门吊梁十二咪五\n4634     龙骨碎玻璃                                            龙骨碎玻璃装袋\n\n[4635 rows x 2 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>core_word</th>\n      <th>bad_case_name</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td></td>\n      <td>在丹江口市边装$$袋装化肥$$包装饲料$$散装粮食$$南瓜卸货$$散石油焦公司一手卸车现金$...</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>115挖机</td>\n      <td>115挖机机马上走</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>120挖机</td>\n      <td>120挖机机马上走$$120挖机挖机机马上走</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>125每吨</td>\n      <td>好铁沙125每吨</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>1304车头</td>\n      <td>1304车头1台</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>4630</th>\n      <td>龙船摆件</td>\n      <td>3米龙船摆件</td>\n    </tr>\n    <tr>\n      <th>4631</th>\n      <td>龙虾</td>\n      <td>厢装龙虾</td>\n    </tr>\n    <tr>\n      <th>4632</th>\n      <td>龙虾尾</td>\n      <td>龙虾尾冷运</td>\n    </tr>\n    <tr>\n      <th>4633</th>\n      <td>龙门吊梁</td>\n      <td>龙门吊梁十二咪五</td>\n    </tr>\n    <tr>\n      <th>4634</th>\n      <td>龙骨碎玻璃</td>\n      <td>龙骨碎玻璃装袋</td>\n    </tr>\n  </tbody>\n</table>\n<p>4635 rows × 2 columns</p>\n</div>"
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def join(df):\n",
    "    return \"$$\".join(df.values)\n",
    "df = origin_df.groupby(['core_word'])[\"bad_case_name\"].apply(join)\n",
    "df = df.to_frame()\n",
    "df = df.reset_index()\n",
    "df"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "outputs": [
    {
     "data": {
      "text/plain": "     core_word  bad_case_name\n0                       41540\n1        115挖机              1\n2        120挖机              2\n3        125每吨              1\n4       1304车头              1\n...        ...            ...\n4630      龙船摆件              1\n4631        龙虾              1\n4632       龙虾尾              1\n4633      龙门吊梁              1\n4634     龙骨碎玻璃              1\n\n[4635 rows x 2 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>core_word</th>\n      <th>bad_case_name</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td></td>\n      <td>41540</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>115挖机</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>120挖机</td>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>125每吨</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>1304车头</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>4630</th>\n      <td>龙船摆件</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>4631</th>\n      <td>龙虾</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>4632</th>\n      <td>龙虾尾</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>4633</th>\n      <td>龙门吊梁</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>4634</th>\n      <td>龙骨碎玻璃</td>\n      <td>1</td>\n    </tr>\n  </tbody>\n</table>\n<p>4635 rows × 2 columns</p>\n</div>"
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "h = origin_df.groupby(['core_word'])[\"bad_case_name\"].count()\n",
    "h = h.to_frame()\n",
    "h = h.reset_index()\n",
    "h"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'DataFrame' object has no attribute 'to_e'",
     "output_type": "error",
     "traceback": [
      "\u001B[1;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[1;31mAttributeError\u001B[0m                            Traceback (most recent call last)",
      "Input \u001B[1;32mIn [6]\u001B[0m, in \u001B[0;36m<module>\u001B[1;34m\u001B[0m\n\u001B[0;32m      3\u001B[0m result\u001B[38;5;241m.\u001B[39mto_csv(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mcore_word.csv\u001B[39m\u001B[38;5;124m\"\u001B[39m)\n\u001B[0;32m      4\u001B[0m result\n\u001B[1;32m----> 5\u001B[0m \u001B[43mresult\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mto_e\u001B[49m\n",
      "File \u001B[1;32mD:\\Users\\fengfeng.qiu\\Anaconda3\\envs\\Validate\\lib\\site-packages\\pandas\\core\\generic.py:5583\u001B[0m, in \u001B[0;36mNDFrame.__getattr__\u001B[1;34m(self, name)\u001B[0m\n\u001B[0;32m   5576\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m (\n\u001B[0;32m   5577\u001B[0m     name \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;129;01min\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_internal_names_set\n\u001B[0;32m   5578\u001B[0m     \u001B[38;5;129;01mand\u001B[39;00m name \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;129;01min\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_metadata\n\u001B[0;32m   5579\u001B[0m     \u001B[38;5;129;01mand\u001B[39;00m name \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;129;01min\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_accessors\n\u001B[0;32m   5580\u001B[0m     \u001B[38;5;129;01mand\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_info_axis\u001B[38;5;241m.\u001B[39m_can_hold_identifiers_and_holds_name(name)\n\u001B[0;32m   5581\u001B[0m ):\n\u001B[0;32m   5582\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28mself\u001B[39m[name]\n\u001B[1;32m-> 5583\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;43mobject\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[38;5;21;43m__getattribute__\u001B[39;49m\u001B[43m(\u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mname\u001B[49m\u001B[43m)\u001B[49m\n",
      "\u001B[1;31mAttributeError\u001B[0m: 'DataFrame' object has no attribute 'to_e'"
     ]
    }
   ],
   "source": [
    "result = pd.merge(h,df,on='core_word')\n",
    "result.drop([0],inplace=True)\n",
    "result.to_csv(\"core_word.csv\")\n",
    "result\n",
    "result.to_e"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}